# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 
# Message:  长安官网所有车型数据抓取
import json
from untils.requestsTools import requestsTools
from untils.MysqlTools import MysqlTools
import pymongo
import os
from setting import *
import time
from _set import _set
import hashlib


class CaSpider:
    def __init__(self):
        self.basePath = os.getcwd() + os.sep + '../oushangHtml/new'
        if not os.path.exists(self.basePath):
            os.makedirs(self.basePath)
        self.requestsTools = requestsTools(self.basePath)
        self.requestsTools.headers = HEADERS
        self.requestsTools.cookies = COOKIES
        self.imgPath = os.getcwd() + os.sep + '../oushangImg/new'
        if not os.path.exists(self.imgPath):
            os.makedirs(self.imgPath)
        self._set = _set
        self.colDict = {
            "carsinfo": self.get_monCol("cars_info"),
            "group": self.get_monCol("cars_group"),
            "struc": self.get_monCol("cars_struc"),
            "parts": self.get_monCol("cars_parts")
        }
        db_config = dict(
            host="192.168.10.166",
            port=3307,
            user="a",
            passwd="110",
            db="changan_new",
            charset="utf8",
        )
        self.sqlDb = MysqlTools(db_config=db_config, type='dict', auto_commit=5000)

    def get_monCol(self, col):
        monClient = pymongo.MongoClient(host='192.168.10.222', port=27017)
        monDb = monClient["changan_local"]
        monCol = monDb[col]
        return monCol

    def get_carCode(self, isOld=0):
        monCol = self.get_monCol("carCode_clean")
        data = monCol.find({"isOld": isOld})
        return list(data)

    def get_timeStr(self, len=13):
        return str(time.time()).replace('.', '')[:len]

    def urlJoin(self, url, params):
        pass

    def get_scid(self, scCode):
        url = 'http://epc.changan.com.cn:8080/record/rcd-sc/selectScIdByScCode'
        params = '?{}={}&{}={}'.format("scCode", scCode, "_", self.get_timeStr())
        url = url + params
        print("请求scid: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scCode)
        # print("debug")
        print(responseInfo.get("content"))
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", {}).get("scId", "")

    def get_groupContent_l(self, scid):
        url = 'http://epc.changan.com.cn:8080/record/rcd-system/selectSysByScId'
        params = '?{}={}&{}={}'.format("scId", scid, "_", self.get_timeStr())
        url = url + params
        print("请求主组信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_strucContent_l(self, scid, systemId):
        url = 'http://epc.changan.com.cn:8080/record/rcd-module/getBySysIdAndScId'
        params = '?{}={}&{}={}&{}={}'.format("systemId", systemId, "scId", scid, "_", self.get_timeStr())
        url = url + params
        print("请求分组信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid+systemId)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_partsContent_l(self, scId, systemId, moduleId, mvId):
        url = 'http://epc.changan.com.cn:8080/record/rcd-component/listByModuleIdAndSystemIdAndScId'
        params = "?{}={}&{}={}&{}={}&{}={}".format("moduleId", moduleId,
                                                    "scId", scId,
                                                    "systemId", systemId,
                                                    "mvId", mvId)
        url = url + params
        print("请求零件信息: ", url)
        responseInfo = self.requestsTools.parseUrl(url, fileName=scId+systemId+moduleId+mvId)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])


    def get_img_info(self, scid, rmmId):
        # 获取图片信息
        url = 'http://epc.changan.com.cn:8080//record/rcd-module/queryModuleImgByRmmid'
        print("请求图片信息: ", url)
        params = '?{}={}&{}={}'.format("rmmId", rmmId, "_", self.get_timeStr())
        url = url + params
        responseInfo = self.requestsTools.parseUrl(url, fileName=scid+rmmId)
        responseContent = json.loads(responseInfo["content"])
        return responseContent.get("data", [])

    def get_pdf(self, carCode_fina, moduleCode, pdfName, suffix):
        url = 'http://epc.changan.com.cn:8080/front/download/{}'.format(pdfName+'.'+suffix)
        print("请求pdf文件: ", url)
        # 获取filePath
        _scidPath = self.imgPath + os.sep + carCode_fina
        if not os.path.exists(_scidPath):
            os.makedirs(_scidPath)
        _modulePath = _scidPath + os.sep + moduleCode
        if not os.path.exists(_modulePath):
            os.makedirs(_modulePath)
        _pdfPath = _modulePath + os.sep + pdfName + '.' + suffix
        if os.path.exists(_pdfPath):
            print("pdf 已存在")
            return
        else:
            pdfResponse = self.requestsTools.getImg(url)
            if pdfResponse:
                with open(_pdfPath, 'wb') as f:
                    f.write(pdfResponse.content)

    def _get_md5(self, str):
        '''获取md5加密电子指纹'''
        _md5 = hashlib.md5()
        _md5.update(str)
        return _md5.hexdigest()

    def save2mongo(self, _dict, ctype):
        col = self.colDict[ctype]
        # col.insert(_dict)

    def save2sql(self, _dict, ctype):
        tableDict = {
            "carsinfo": "cars_info_collection_ous_new",
            "group": "cars_group_collection_ous_new",
            "struc": "cars_structure_collection_ous_new",
            "parts": "cars_part_num_ous_new"
        }
        table = tableDict[ctype]
        inSql = self.sqlDb.get_insert_sql(table, _dict)
        try:
            self.sqlDb.execute_sql(inSql)
        except Exception as e:
            print(e.__repr__())
            print(inSql)


    def save_info(self, carCode_info):
        carCode = carCode_info.get("carCode")
        carCode_fina = carCode_info.get("carCode_fina")
        carName = carCode_info.get("carName")
        carTail_fina = carCode_info.get("carTail_fina")
        _str = carName + carCode + carTail_fina + carCode_fina.replace("-", "_").replace(".", "_")
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "name": "欧尚",
            "brand": "changan",
            "brandCode": "changan",
            "condition": carName,
            "limit": carTail_fina,
            "limit2": carCode,
            "limit3": carCode_fina,
            "limit4": "",
            "label": "",
            "code": carCode_fina.replace("-", "_").replace(".", "_"),
            "description": "",
            "imagePath": "",
            "imgName": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 2,
        }
        self.save2sql(_dict, 'carsinfo')


    def save_group(self, carCode_fina, group_num, groupContent):
        systemNm = groupContent.get("systemNm")
        carCode_fina = carCode_fina.replace("-", "_").replace(".", "_")
        _str = carCode_fina + str(group_num) + systemNm
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "cid": carCode_fina,
            "num": str(group_num),
            "brandCode": "changan",
            "label": systemNm,
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
            # "": "",
        }
        self.save2sql(_dict, 'group')


    def save_struc(self, carCode_fina, group_num, strucContent):
        moduleCode = strucContent.get("moduleCode").replace("-", "_")
        moduleNm = strucContent.get("moduleNm")
        carCode_fina = carCode_fina.replace("-", "_").replace(".", "_")
        _str = carCode_fina + str(group_num) + moduleCode + moduleNm
        _dict = {
            "uid": self._get_md5(_str.encode()),
            "cid": carCode_fina,
            "num": str(group_num),
            "maingroup": str(group_num),
            "subgroup": moduleCode,
            "mid": moduleCode,
            "brandCode": "changan",
            "name": moduleNm,
            "description": "",
            "model": "",
            "condition": "",
            "imgName": "",
            "imageLarge": "",
            "imgPoints": "",
            "logo": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2sql(_dict, 'struc')


    def save_parts(self, carCode_fina, group_num, moduleCode, psort, partsContent):
        compCode = partsContent.get("compCode") if partsContent.get("compCode") else ""  # 备件号
        partNumber = partsContent.get("partNumber") if partsContent.get("partNumber") else ""  # 零件号
        compNmCn = partsContent.get("compNmCn") if partsContent.get("compNmCn") else ""  # 零件名称
        compAmount = partsContent.get("compAmount")  # 零件数量
        imageNo = partsContent.get("imageNo")  # 零件pnum
        enableTime = partsContent.get("enableTime") if partsContent.get("enableTime") else ""  # 开始时间
        disableTime = partsContent.get("disableTime") if partsContent.get("disableTime") else ""  # 结束时间
        carCode_fina = carCode_fina.replace("-", "_").replace(".", "_")
        moduleCode = moduleCode.replace("-", "_")
        _str = carCode_fina + str(group_num) + moduleCode + str(psort) + partNumber + compCode + compNmCn
        _dict = {
            "cid": carCode_fina,
            "uid": self._get_md5(_str.encode()),
            "num": group_num,
            "maingroup": group_num,
            "subgroup": moduleCode,
            "mid": moduleCode,
            "brandCode": "changan",
            "pnum": imageNo,
            "itid": imageNo,
            "psort": psort,
            "pid": partNumber,
            "label": compNmCn,
            "remark": enableTime.replace("-", "") + "-" + disableTime.replace("-", ""),
            "quantity": str(compAmount) if compAmount else "",
            "model": compCode,
            "isreplace": "",
            "replacelabel": "",
            "rid": "",
            "condition": "",
            "img": "",
            "logo": "",
            "price": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2sql(_dict, 'parts')

    def _getData(self, carCode_info):
        carCode = carCode_info.get("carCode")
        carCode_fina = carCode_info.get("carCode_fina")
        carName = carCode_info.get("carName")
        carTail_fina = carCode_info.get("carTail_fina")
        if carCode_fina in self._set:
            print("{} 已抓取过".format(carCode_fina))
            return
        # TODO 保存cars_info表
        self.save_info(carCode_info)
        return
        # 第一次请求, 获取scid
        scid = self.get_scid(carCode_fina)
        # 第二次请求, 获取主组信息
        groupContent_l = self.get_groupContent_l(scid)
        group_num = 0
        for groupContent in groupContent_l:
            group_num += 1
            print(groupContent)
            # TODO 保存group表
            self.save_group(carCode_fina, group_num, groupContent)

            systemId = groupContent.get("systemId", "")
            if not systemId:
                continue
            strucContent_l = self.get_strucContent_l(scid, systemId)
            for strucContent in strucContent_l:
                rmmId = strucContent.get("rmmId", "")
                moduleCode = strucContent.get("moduleCode", "")
                moduleId = strucContent.get("moduleId", "")
                systemId = strucContent.get("systemId", "")
                mvId = strucContent.get("mvId", "")

                # TODO 保存struc表
                self.save_struc(carCode_fina, group_num, strucContent)

                if rmmId and moduleCode:  # 请求图片信息
                    imgInfo_l = self.get_img_info(scid, rmmId)
                else:
                    imgInfo_l = []
                partsContent_l = self.get_partsContent_l(scid, systemId, moduleId, mvId)
                psort = 0
                for partsContent in partsContent_l:
                    partNum = partsContent.get("partNumber", "")
                    compNmCn = partsContent.get("compNmCn", "")
                    print(partNum, compNmCn)
                    # TODO 保存零件表
                    psort += 1
                    self.save_parts(carCode_fina, group_num, moduleCode, psort, partsContent)

                if imgInfo_l:
                    for i in range(len(imgInfo_l)):
                        imgInfo = imgInfo_l[i]
                        pdfName = imgInfo.get("id")
                        suffix = imgInfo.get("suffix")
                        self.get_pdf(carCode_fina, moduleCode, pdfName, suffix)
        self._set.append(carCode_fina)


    def getData(self):
        # 获取新车型编号
        carCode_info_l = self.get_carCode()
        for carCode_info in carCode_info_l:
            # carCode_fina = carCode_info.get("carCode_fina")
            self._getData(carCode_info)
            self.sqlDb.commit_sql()
            # return

    def run(self):
        try:
            self.getData()
            with open('./_set.py', 'w') as s:
                s.write("_set = {}".format(self._set))
        except Exception as e:
            with open('./_set.py', 'w') as s:
                s.write("_set = {}".format(self._set))
            print(e.__repr__())



    #  获取欧尚数据
    def get_carsModel(self):
        postData = {
            "brandId": "474549e1fee84369913723719d5711f2"
        }
        url = 'http://epc.changan.com.cn:8080/system/bus-style/listBusStyleByBrandId'
        responseInfo = self.requestsTools.parseUrl(url=url, method='POST', postData=postData, fileName=postData["brandId"])
        data = json.loads(responseInfo["content"])["data"]
        _str = ','.join([i['styleId'] for i in data])

        # 请求车系数据
        url = 'http://epc.changan.com.cn:8080/system/bus-series/selectSeriesByStyleId'
        params = '?{}={}&{}={}'.format("styleIds", _str, "_", self.get_timeStr())
        url = url + params
        print("请求车系数据: ", url)
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=_str)
        return json.loads(responseInfo["content"])["data"]

    def get_carsCode(self, seriesId):
        url = '''http://epc.changan.com.cn:8080/record/rcd-sc/searchScCodePreBySeriesId'''
        params = '?{}={}&{}={}'.format("seriesId", seriesId, "_", self.get_timeStr())
        url = url + params
        print("请求车型号")
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=seriesId)
        data = json.loads(responseInfo["content"])["data"]
        return data

    def get_carsStatuCode(self, scCodePreFix):
        url = '''http://epc.changan.com.cn:8080/record/rcd-sc/selectScChNameByScCodePre'''
        params = '?{}={}&{}={}'.format("scCodePreFix", scCodePreFix, "_", self.get_timeStr())
        url = url + params
        print("请求车型状态号: ", url)
        responseInfo = self.requestsTools.parseUrl(url=url, fileName=scCodePreFix)
        data = json.loads(responseInfo["content"])["data"]
        return data

    def get_oushangData(self):
        # 获取欧尚车系数据
        carsModel_data = self.get_carsModel()
        for carsModel in carsModel_data:
            print(carsModel)
            seriesId = carsModel.get("seriesId")
            # 获取车型号
            carsCode_l = self.get_carsCode(seriesId)
            for carsCode in carsCode_l:
                print(carsCode)
                carsStatusCode_l = self.get_carsStatuCode(carsCode)
                for carsStatusCode_info in carsStatusCode_l:
                    code = carsStatusCode_info["code"]
                    name = carsStatusCode_info["name"].strip()
                    scId = carsStatusCode_info["scId"].strip()
                    carCode_info = {
                        "carCode": carsCode,
                        "carName": "COSMOS",
                        "carTail_fina": name,
                        "carCode_fina": code,
                    }
                    print(carCode_info)
                    self._getData(carCode_info)
        self.sqlDb.commit_sql()

if __name__ == '__main__':
    tools = CaSpider()
    tools.get_oushangData()

